{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensorflow2教程-rnn变体"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.导入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_words = 30000\n",
    "maxlen = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=num_words)\n",
    "print(x_train.shape, ' ', y_train.shape)\n",
    "print(x_test.shape, ' ', y_test.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen, padding='post')\n",
    "x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen, padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "print(x_train.shape, ' ', y_train.shape)\n",
    "print(x_test.shape, ' ', y_test.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 200, 32)           960000    \n",
      "_________________________________________________________________\n",
      "unified_lstm_4 (UnifiedLSTM) (None, 200, 32)           8320      \n",
      "_________________________________________________________________\n",
      "unified_lstm_5 (UnifiedLSTM) (None, 1)                 136       \n",
      "=================================================================\n",
      "Total params: 968,456\n",
      "Trainable params: 968,456\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "def lstm_model():\n",
    "    model = keras.Sequential([\n",
    "        layers.Embedding(input_dim=30000, output_dim=32, input_length=maxlen),\n",
    "        layers.LSTM(32, return_sequences=True),\n",
    "        layers.LSTM(1, activation='sigmoid', return_sequences=False)\n",
    "    ])\n",
    "    model.compile(optimizer=keras.optimizers.Adam(),\n",
    "                 loss=keras.losses.BinaryCrossentropy(),\n",
    "                 metrics=['accuracy'])\n",
    "    return model\n",
    "model = lstm_model()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 22500 samples, validate on 2500 samples\n",
      "Epoch 1/5\n",
      "22500/22500 [==============================] - 100s 4ms/sample - loss: 0.6589 - accuracy: 0.6056 - val_loss: 0.6090 - val_accuracy: 0.6544\n",
      "Epoch 2/5\n",
      "22500/22500 [==============================] - 100s 4ms/sample - loss: 0.6418 - accuracy: 0.6076 - val_loss: 0.6356 - val_accuracy: 0.6068\n",
      "Epoch 3/5\n",
      "22500/22500 [==============================] - 100s 4ms/sample - loss: 0.5763 - accuracy: 0.6980 - val_loss: 0.5984 - val_accuracy: 0.6724\n",
      "Epoch 4/5\n",
      "22500/22500 [==============================] - 102s 5ms/sample - loss: 0.5749 - accuracy: 0.7150 - val_loss: 0.5044 - val_accuracy: 0.7900\n",
      "Epoch 5/5\n",
      "22500/22500 [==============================] - 102s 5ms/sample - loss: 0.4893 - accuracy: 0.7993 - val_loss: 0.7230 - val_accuracy: 0.6756\n",
      "CPU times: user 14min 29s, sys: 42.5 s, total: 15min 12s\n",
      "Wall time: 8min 24s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "history = model.fit(x_train, y_train, batch_size=64, epochs=5,validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xd8FHX6wPHPk04gQOgQSqjSBRIiRZQqTYqFptIUuBOQs/4Ez8rhyZ2eDQEPpBcBQaQqRUFFaQm9E3qoAUIn/fv7YxYuhEA2yWY3mzzv1ysvdme+851nlmSenZlvEWMMSimllIerA1BKKZUzaEJQSikFaEJQSillowlBKaUUoAlBKaWUjSYEpZRSgCYEpZRSNpoQlFJKAZoQlFJK2Xi5OoCMKFasmAkODnZ1GEop5VYiIiLOG2OKp1fOrRJCcHAw4eHhrg5DKaXciogcs6ec3jJSSikFaEJQSillowlBKaUU4GbPENKSkJBAVFQUsbGxrg4lV/Dz86Ns2bJ4e3u7OhSllJPZlRBEpB3wBeAJfGOMGZ1qfXlgGlDYVma4MWa5bd0I4AUgCRhmjFlhT532ioqKIiAggODgYEQkM1UoG2MMFy5cICoqiooVK7o6HKWUk6V7y0hEPIGxQHugJtBLRGqmKvY2MM8YUx/oCYyzbVvT9r4W0A4YJyKedtZpl9jYWIoWLarJwAFEhKJFi+rVllJ5lD3PEMKASGPMYWNMPDAH6JKqjAEK2l4XAk7ZXncB5hhj4owxR4BIW3321Gk3TQaOo5+lUnmXPQkhCDiR4n2UbVlK7wPPiUgUsBx4KZ1t7alTKaXyvAvX4vhgyW5iE5KyfV+OamXUC5hqjCkLdABmiIhD6haRQSISLiLh0dHRjqjSoS5dusS4ceMyvF2HDh24dOnSfcu8++67rF69OrOhKaXc3I34RJ6fFs7sjcc5FH0t2/dnz0n7JFAuxfuytmUpvQDMAzDGrAf8gGL32daeOrHVN8EYE2qMCS1ePN2e1053r4SQmJh43+2WL19O4cKF71tm5MiRtG7dOkvxKaXcU2JSMi/N3srOqEuM6VWfWmUKZfs+7UkIm4GqIlJRRHywHhIvTlXmONAKQERqYCWEaFu5niLiKyIVgarAJjvrdAvDhw/n0KFD1KtXj4YNG9KsWTM6d+5MzZrWM/KuXbsSEhJCrVq1mDBhwu3tgoODOX/+PEePHqVGjRoMHDiQWrVq8dhjj3Hz5k0A+vXrx/z582+Xf++992jQoAF16tRh3759AERHR9OmTRtq1arFgAEDqFChAufPn3fyp6CUciRjDO8u3s3P+87xQZfaPFarlFP2m26zU2NMoogMBVZgNRGdbIzZLSIjgXBjzGLgNWCiiLyC9YC5nzHGALtFZB6wB0gEhhhjkgDSqjOrB/PBkt3sOXUlq9XcoWaZgrzXqdY9148ePZpdu3axbds21q5dS8eOHdm1a9ftZpuTJ0+mSJEi3Lx5k4YNG/LUU09RtGjRO+o4ePAg3377LRMnTqR79+4sWLCA55577q59FStWjC1btjBu3Dg++eQTvvnmGz744ANatmzJiBEj+Omnn5g0aZJDj18p5Xzj1h5i9sbjvNi8Mr0bVXDafu3qh2DrU7A81bJ3U7zeAzS9x7YfAh/aU2duEBYWdkcb/i+//JKFCxcCcOLECQ4ePHhXQqhYsSL16tUDICQkhKNHj6ZZ95NPPnm7zPfffw/AunXrbtffrl07AgMDHXo8SinnWhARxccr9vNE/SD+r+0DTt232/dUTul+3+SdJX/+/Ldfr127ltWrV7N+/Xr8/f1p3rx5mm38fX19b7/29PS8fcvoXuU8PT3TfUahlHI/vx+M5s0FO2hapSj/eqqu05uB61hGWRQQEMDVq1fTXHf58mUCAwPx9/dn3759bNiwweH7b9q0KfPmzQNg5cqVxMTEOHwfSqnst/vUZV6cuYUqJQow/rkQfLycf3rOVVcIrlC0aFGaNm1K7dq1yZcvHyVLlry9rl27dnz99dfUqFGDBx54gEaNGjl8/++99x69evVixowZNG7cmFKlShEQEODw/Silsk9UzA36T9lMQT8vpvYPo6Cfa8YSE+vZr3sIDQ01qSfI2bt3LzVq1HBRRK4XFxeHp6cnXl5erF+/nhdffJFt27Zlqc68/pkq5UyXbsTz9NfrOXsllgUvNqFaScd/oRORCGNMaHrl9ArBzR0/fpzu3buTnJyMj48PEydOdHVISt3f9rmw7lPovRAKlnF1NC4Vm5DEoOkRHL9wg+kvhGVLMsgITQhurmrVqmzdutXVYShlnyunYfnrEHcFlr0GPWdDHh0/KznZ8Nq87Ww6epExverTqFLR9DfKZvpQWSnlPD++AUnxEDYI9i+HXQtcHZHLfLh8L8t2nubvHWrQ6cGccaWkCUEp5Rx7l8LeJfDom9D2IyjTAH78P7ie93rWf/P7YSatO0L/psEMaJZz5h7RhKCUyn6xV2D5G1CyNjR5CTy9oMtYa/mPb7o6OqdauuMUo5btpX3tUrzdsWaOGnJeE4JSKvv9PBKunoZOX4KnrUllyZrwyOuwaz7s/9G18TnJxsMXeHXudhoGB/JZj3p4euScZACaEJyuQIECAJw6dYqnn346U3Vs27aN5cv/N+rH4sWLGT06UzOQKpX9TmyCzd/AQ3+BsiF3rnv4VShRC5a+AjfvPxy8uzt49ioDp4dTrkg+JvYJxc/b09Uh3UUTgouUKVPm9kimGZU6IXTu3Jnhw4c7KjSlHCcxHhYPg4JB0PLtu9d7+UCXMXDtLKx69+71ucTZK7H0m7IZX29PpvYPo7C/j6tDSpMmhCwaPnw4Y8eOvf3+/fffZ9SoUbRq1er2UNWLFi26a7ujR49Su3ZtABo1asTu3f8b7LV58+aEh4ezadMmGjduTP369WnSpAn79+8nPj6ed999l7lz51KvXj3mzp3L1KlTGTp06O16W7ZsSd26dWnVqhXHjx8HrKG0hw0bRpMmTahUqVKmk5FSGfLnFxC9Fzp+Ar73aGMfFAKNh8KWaXD4V+fG5wRXYxPoN2Uzl27EM6VfQ8oV8Xd1SPeUu/oh/Dgczux0bJ2l6kD7e9+O6dGjBy+//DJDhgwBYN68eaxYsYJhw4ZRsGBBzp8/T6NGjejcufM9Hx716NGDefPm8cEHH3D69GlOnz5NaGgoV65c4ffff8fLy4vVq1fz1ltvsWDBAkaOHEl4eDhfffUVAFOnTr1d10svvUTfvn3p27cvkydPZtiwYfzwww8AnD59mnXr1rFv3z46d+6c6VtWStnlfCT8+jHU7AoPtL9/2RZvwb6lsPglGLwefPLfv7ybiE9M5sWZWzh49iqT+zWkdlD2T3KTFXqFkEX169fn3LlznDp1iu3btxMYGEipUqV46623qFu3Lq1bt+bkyZOcPXv2nnV079799jf2efPm3T5RX758mW7dulG7dm1eeeWVO64i7mX9+vU888wzAPTu3Zt169bdXte1a1c8PDyoWbPmfeNRKsuMgaUvg5cftP93+uW980Hnr+DSMfhlVPbH5wTGGIYv2MG6yPN89GQdHqmW82Z8TC13XSHc55t8durWrRvz58/nzJkz9OjRg1mzZhEdHU1ERATe3t4EBwenOez1LUFBQRQtWpQdO3Ywd+5cvv76awDeeecdWrRowcKFCzl69CjNmzfPUpwph9l2pzGslBvaOhOO/g6dvoCAkumXBwhuCqEvwIbxUOsJKBeWvTFms09W7uf7rSd5rU01uoWWS3+DHMCuKwQRaSci+0UkUkTuenopIp+JyDbbzwERuWRb3iLF8m0iEisiXW3rporIkRTr6jn20JynR48ezJkzh/nz59OtWzcuX75MiRIl8Pb2Zs2aNRw7dsyuOv79739z+fJl6tatC1hXCEFBQcCdt4XuN+R2kyZNmDNnDgCzZs2iWbNmWTw6pTLo2jlY+TZUaAr1+2Rs29bvWw+gFw2FxLjsiM4pZm44xtg1h+gVVo6hLau4Ohy7pZsQRMQTGAu0B2oCvUSkZsoyxphXjDH1jDH1gDHA97bla1IsbwncAFam2PSNW+uNMVkbotOFatWqxdWrVwkKCqJ06dI8++yzhIeHU6dOHaZPn0716tXTrePpp59mzpw5dO/e/fay//u//2PEiBHUr1//jglxWrRowZ49e24/VE5pzJgxTJkyhbp16zJjxgy++OILxx2oUvb4aTgk3IDHPwePDN6V9itoXVWc3w+/2nGrKQdatecs7y7aRcvqJfhHl9o5quNZetId/lpEGgPvG2Pa2t6PADDGfHSP8n8C7xljVqVaPgh41BjzrO39VGCpMcbu5i46/LVz6GeqMu3ASpjdDZq/Bc2z0AN54V9hxzwYtBZK13VUdNlu6/EYek3cQLWSAcwZ1Ah/n5xxV97e4a/tSd9BwIkU76Nsy9LaaQWgIvBLGqt7At+mWvahiOyw3XLyTWMbpZS7iLsGy16F4tXh4VeyVlfbf4J/UVg0BJISHBNfNjty/jovTAunRIAfk/s1zDHJICMc3cqoJzDfGJOUcqGIlAbqACtSLB4BVAcaAkWANL9OiMggEQkXkfDo6GgHh6uUcpg1/4TLJ6xbPl5Z7HjlX8Tqu3BmB/z5pWPiy0bnr8XRb8omjDFMez6MYgXc8/utPQnhJJDyEXlZ27K0pHUVANAdWGiMuZ3qjTGnjSUOmAKk2aTAGDPBGBNqjAktXjztZlvaYsZx9LNUmXJyC2wcD6HPQ3kHTRVbswvU6Axr/wXRBxxTZza4EZ/IC1M3c/ZKLJP6NaRiMfftQ2FPQtgMVBWRiiLig3XSX5y6kIhUBwKB9WnU0YtUicJ21YBYT1y6ArsyFrrFz8+PCxcu6InMAYwxXLhwAT8/P1eHotxJUgIsGQb5S1ithBypwydWH4XFQyE5Kf3yTpaYlMxLs7ey8+RlxvRqQIPyga4OKUvSvclljEkUkaFYt3s8gcnGmN0iMhIIN8bcSg49gTkm1ZlZRIKxrjBS90mfJSLFAQG2AX/NzAGULVuWqKgo9HaSY/j5+VG2bFlXh6HcyYZx1ggB3WeAn4N74gaUhHaj4Ye//m+AvBzCGMM7i3bz875z/KNrbdrUtLO/RQ6WbiujnCStVkZKKRe6eATGNYYqraDnrOzZhzEw62k4tt4a1iKwQvbsJ4O++uUgn6w8wODmlfm/duk3LXclR7YyUkqpuxljDVvt4WXf8BSZJWL1aRCxbk3lgC+x8yOi+GTlAZ6oH8QbbR9wdTgOowlBKZU5O+bB4TXQ+j0olGZLdMcpXA7afACH11rDYrjQbweiGb5gB02rFOVfT9V1q45n6dGEoJTKuOsXYMUIKBtmjT/kDCHPW8NhrPg7XDntnH2msuvkZV6cGUGVEgUY/1wIPl656xSau45GKeUcK/8OsZetPgcZHZ4iszw8oPMYSIqDZa85/dZRVMwN+k/dTKF83kztH0ZBP2+n7t8ZNCEopTLm0BrY/i00fdmaF9mZila25k7Yvwx2f++03V66EU/fyZuIS0hi6vNhlCqUO5tma0JQStkv/oY1z0GRyvDIG66JodEQKFMflv+fdesqm8UmJDFwejgnLt5kQp9QqpW8x8xvuYAmBKWU/X79F8QctW4VebvoW7KnF3QZa92y+ikLA+jZITnZ8Oq8bWw+GsN/uj9Io0pFs3V/rqYJQSlln9M74M8xUL83VHTxPBsla0Gz12Dnd7D/p2zbzahle1m+8wxvd6xBpwfLZNt+cgpNCEqp9CUnWX0A/ItAm5GujsbS7DUoUdPqCxF72eHVf/P7YSb/cYT+TYN54eGKDq8/J9KEoJRK36YJcGqrNYyEfxFXR2Px8oEuX8G1M7DyHYdWvWT7KUYt20uHOqV4p2PNXNXX4H40ISil7u/Scfj5H1D1Maj9lKujuVNQCDQeAlumweHUw6VlzobDF3ht3nYaBgfyafd6eHjkjWQAmhCUUvdjDCx7HTDQ8T/W8BE5TfO3oEgl65ZW/PUsVXXg7FUGTQ+nXJF8TOwTip+3p4OCdA+aEJRS97b7ezi4Alq+DYXLuzqatPn4Q+evrNZPv3yY6WrOXI6l3+RN+Hp7MrV/GIX9szjJjxvShKCUStvNGPjxTavN/0OZGp3eeYKbWkNobBgHJzZnePOrsQn0m7KJyzcTmNKvIeWK+GdDkDmfJgSlVNpWvQs3LkKnL8HDDW6dtH4fCgZZ8zAnxtm9WXxiMn+dGUHkuWuMfy6E2kEOntPBjWhCUErd7eg62DIdmgyF0nVdHY19/ApCp8/h/H747WO7NjHG8OaCHfwReYHRT9XlkWppT9ObV2hCUErdKSEWlvwNCleAR4e7OpqMqdoG6vaEdZ9Zs7il4+MV+1m49SSvtanG0yE6U6BdCUFE2onIfhGJFJG7fkNE5DMR2Wb7OSAil1KsS0qxbnGK5RVFZKOtzrm2+ZqVUq72+3/gQqT1bdvHDe+lt/sI8gVat46SEu9ZbMaGY4xbe4heYeUZ2rKKEwPMudJNCCLiCYwF2gM1gV4icscQh8aYV4wx9Ywx9YAxQMphCG/eWmeM6Zxi+b+Az4wxVYAYwEmDqiul7uncXuvbdd2eULmlq6PJHP8i0OETOL0d/vwyzSIrd5/hvUW7aFW9BP/oUivPdDxLjz1XCGFApDHmsDEmHpgDdLlP+V7At/erUKxPvyUw37ZoGtDVjliUUtklORkWDwPfAGib+eabOUKtrlCjE6wdDecP3rFqy/EYhs3ZSp2gQox5pj5ennrn/BZ7Pokg4ESK91G2ZXcRkQpAReCXFIv9RCRcRDaIyK2TflHgkjHm1vXcPetUSjlJ+CSI2gRt/wn5i7k6mqzr8B/wzgeLhlrJDjhy/joDpoVTsqAfk/o1xN/Hy8VB5iyOTo09gfnGmKQUyyoYY0KBZ4DPRaRyRioUkUG2hBIeHR3tyFiVUrdcOQWrP4BKzeHBnq6OxjECSlrPE05sgM0TOX8tjr6TNwEwrX8YxQr4ujjAnMeehHASKJfifVnbsrT0JNXtImPMSdu/h4G1QH3gAlBYRG6l53vWaYyZYIwJNcaEFi+et5uEKZVtlr8ByYnw+Gc5c3iKzHqwF1RpjVn9ASMmLeXc1Vgm9Q0luFh+V0eWI9mTEDYDVW2tgnywTvqLUxcSkepAILA+xbJAEfG1vS4GNAX2GGMMsAZ42la0L7AoKweilMqkvUtg31JoPtwaEyg3ESGxw6fEJSbT58JnjOlZn/rlA10dVY6VbkKw3ecfCqwA9gLzjDG7RWSkiKRsNdQTmGM72d9SAwgXke1YCWC0MWaPbd2bwKsiEon1TGFS1g9HKZUhsZetq4OSdaxRQ3MZYwzvrL3MqPieNPPYSZv41a4OKUez64mKMWY5sDzVsndTvX8/je3+BOrco87DWC2YlFKu8vNIuHYWes4CT29XR+NwX/0SybebTjD40UFw5gCseAuqtIaAUq4OLUfS9lZK5VXHN8LmSdbAdUEhro7G4b4LP8F/Vh3gyfpBvNGuhjUmU2IcLHvNGtZb3UUTglJ5UWK8NX9AobLQ4u+ujsbhfj0QzYjvd/JwlWKMfqqu1fGsWBVoPsJ6XrJ7oatDzJE0ISiVF/3xOUTvsya98S3g6mgcatfJywyeGUGVEgUY/1wDfLxSnOYaD4XS9aznJtcvuC7IHEoTglJ5zfmD1migtZ6Eam1dHY1Dnbh4g/5TN1MonzfTng8jwC/VcxFPL+gyFmIvwU9uNnCfE2hCUCovSU62RjL1zgftRrs6Goe6dCOeflM2EZeQxNTnwyhZ0C/tgqVqQ7PXYOc8OLDCuUHmcJoQlMpLts6AY3/AY6Osnry5RGxCEgOnh3Pi4k0m9AmlWsmA+2/Q7HUoXgOWvGw1vVWAJgSl8o6rZ2HVO1DhYajf29XROExysuGVudvYfDSGT3s8SKNKRdPfyMvHunV07Yw1M5wCNCEolXf89KY1+U2nz3PN8BTGGP6xbA8/7jrD2x1r8HjdMvZvXDYEGg2GiKlw5Ldsi9GdaEJQKi/Y/5PV1PKRN6BYVVdH4zCT1h1hyh9Heb5pRQY0y8SwGy3+bg3XsfgliL/h+ADdjCYEpXK7uGtWZ6zi1aHp31wdjcMs2X6KUcv20qFOKd7uWCNzlfj4Q+cxEHMU1rj5HBAOoAlBqdzul1Fw5aTVU9crd8xUu+HwBV6bt52GwYF82r0eHh5ZuAUW/DCEPg8bxkFUuOOCdEOaEJTKzaIiYOPX0PAFKP+Qq6NxiANnrzJoejjli/ozsU8oft6eWa+09QcQUMaahzkxLuv1uSlNCErlVkkJVp+DgFLQKne0pDlzOZZ+kzfh6+3J1P4NKezvoCsev4LWw/boffDbJ46p0w1pQlAqt1r/FZzdaU0471fI1dFk2ZXYBPpN2cTlmwlM7d+QsoH+jt1B1TZQtyes+xTO7HRs3W5CE4JSudHFw9YE89UfhxqPuzqaLItPTObFmRFEnrvG+OdCqFUmmxJcu48gX6A1D3NSYvrlcxlNCErlNsZYPXA9faDDx66OJsuMMby5YAd/RF5g9FN1eaRaNk6l61/E+sxOb4P1Y7JvPzmUXQlBRNqJyH4RiRSRu0aEEpHPRGSb7eeAiFyyLa8nIutFZLeI7BCRHim2mSoiR1JsV89xh6VUHrZ9Dhz5FVq/BwUz0FErh/p4xX4Wbj3J649V4+mQstm/w5pdrSurNR9ZAwHmIekmBBHxBMYC7YGaQC8RqZmyjDHmFWNMPWNMPWAM8L1t1Q2gjzGmFtAO+FxECqfY9I1b2xljtjngeJTK266ft2YFK/cQhDzv6miybMaGY4xbe4heYeUZ0qKKc3YqYg0L7u1ndVhLTnbOfnMAe64QwoBIY8xhY0w8MAfocp/yvYBvAYwxB4wxB22vTwHngGy83lMqj1vxFsRdhU5fgId73xFeufsM7y3aRavqJfhHl1rWJDfOElAK2n4Ex9fD5m+ct18Xs+c3Jgg4keJ9lG3ZXUSkAlAR+CWNdWGAD3AoxeIPbbeSPhMRX7ujVkrdLfJn2DEXHn4FSmSy524OseV4DMPmbKVO2cKMeaY+Xp4uSG71noHKrWD1+xBzzPn7dwFHf8o9gfnGmKSUC0WkNDAD6G+MuXX9NQKoDjQEigBvplWhiAwSkXARCY+OjnZwuErlEvHXYekrULSqNda/GzscfY0Xpm6mZEE/JvUNxd/HyzWBiPxvIMClL+eJeZjtSQgngXIp3pe1LUtLT2y3i24RkYLAMuDvxpgNt5YbY04bSxwwBevW1F2MMROMMaHGmNDixfVuk1JpWjsaLh2zTmDe95gYxg1EX42j35TNiAjT+odRrICLbxwULg+t34dDv8C22a6NxQnsSQibgaoiUlFEfLBO+otTFxKR6kAgsD7FMh9gITDdGDM/VfnStn8F6ArsyuxBKJWnnd4O68dCgz7WuDxu6kZ8Ii9M28y5q7FM6htKcLH8rg7JEvoClG8CK0bA1TOujiZbpZsQjDGJwFBgBbAXmGeM2S0iI0Wkc4qiPYE5xtxxXdUdeATol0bz0lkishPYCRQDRjngeJTKW5ISYfEw8C8KbUa6OppMS0xKZsisLew6eZmvejWgfvlAV4f0Px4e1oioiXHWqLG5+NaRXTfnjDHLgeWplr2b6v37aWw3E5h5jzpb2h2lUiptm/5rdaJ6eorVw9YNGWN4Z9Eu1uyP5sMnatO6Zg6c2rNYFWg+Ala/B3t+gFpPuDqibOHe7dKUystijllDW1dt69YnqDG/RPLtphMMaVGZZx+q4Opw7q3xUChdD5a/ATcuujqabKEJQSl3ZIx1+wJbJyo3nRLzu/ATfLrqAE82COL1xx5wdTj35+llzcN8MwZ+umvAhlxBE4JS7mjXAohcBa3egcLl0i+fA/16IJoR3+/k4SrFGP1kXed2PMusUrXh4Vet/h4HVro6GofThKCUu7lx0fqGWqYBhA1ydTSZsuvkZQbPjKBqyQDGP9cAHy83OhU98ro1HenSlyH2iqujcSg3+l9QSgGw6h0rKXT+EjwcMFuYE52+fJMl20/Rf+pmCvv7MLV/QwL8vF0dVsZ4+Vq3jq6ehlW5Y+KhW1zUBVAplSlHfoOtM6Hpy1Cqjqujua/EpGT2nblK+NGLRBy/RMTRi5y6HAtAsQK+TO3fkJIF3bQTXdlQaDTYmoSo9lNQsZmrI3IIMW7UpjY0NNSEh+ftSbBVHpZwE8Y3BZMEL64HHwfPGJZFl28msOV4DFuOxRB+NIZtJy5xM8EaxaZUQT9CggMJKR9IaHAgNUoXxNsV4xM5UvwNGN/Eev3inznu/yMlEYkwxoSmV06vEJRyF799AhcPQe8fXH7yMcZw9MINIo7FEHHsIhHHYjhw9hoAnh5CjdIBdA8tS0hwEUIqBBJUOJ9L480WPv5Wh7Vpj8OaD6Hth66OKMs0ISjlDs7uhj8+hwd7QeUWTt99bEISu05eJuJYDOHHrKuAC9fjAQjw86JB+UA61S1DSIVAHixXmPy+eeTUUrEZhPSHDeOg1pNQNsTVEWVJHvlfU8qNJSdZw1P4FYLHnPMtNPpq3B3f/nedvEJ8kjVQcXBRfx59oDihFaxv/1VLFMDDww2ajGaXNiPh4EpYNAT+8qv10NlNaUJQKqcLnwwnw+GJCZC/qMOrT0o2HDx3lfCj1jf/iOMxHLtwAwAfTw/qlC1E/6bBNKgQSEiFQNePQJrT+BWExz+H2d3g9/9Ai7dcHVGmaUJQKie7fBJWfwCVWkDd7g6p8lpcIttPXCL8qHXy33oshqtxiQAUK+BDSIVAnn2oPCEVilA7qCC+Xu7VtNUlqj0GdXtYCaFGZ6sDmxvShKBUTmUMLH8dkhPh8c8yNTyFMYaTl27abv9YrX/2nblCsrGqe6BkAJ3qlSHU9u2/fBF/9+gxnBO1G23Nm7BoCAz42Rrqws24X8RK5RV7l8D+5dY96iIV7dokISmZ3aeuEGF78Bt+7CJnr8QB4O/jSf3yhRnaogohwUWoV64whfK5WaewnMy/CHT4GL7rZ/VPePhlV0eUYZoQlMqJbl6yRtUsVQcaDblnsZjr8Ww5HnO79c+OqEvEJlgPf4MK5+OhikUJDQ60CZP5AAAgAElEQVSkQflAqpcKcM3cxHlJza5Q/XFY+5H1b7Eqro4oQzQhKJUT/fwBXD8Hvb69fevBGMOh6Ou3v/lHHIvhUPR1ALw8hFplCvJMWAVCbLd/ShVy017A7kxso8+ODYPFQ6HfcmuCHTehCUGpnObYegifTGLYi0TEVSBibSQRtgfAl24kAFDY35uQ8oE82aCs1fa/bGHy+ejD3xwhoBS0/af1LCF8EoQNdHVEdrMrIYhIO+ALwBP4xhgzOtX6z4BbvWX8gRLGmMK2dX2Bt23rRhljptmWhwBTgXxYs7H9zbjTOBpKOdjZK7FsPXyWkJ/+SrJHCdqsC+PKbxsAqFQ8P4/VLGn79l+ESsXy5+22/zldvWetIcpXvw/V2kLh8q6OyC7pJgQR8QTGAm2AKGCziCw2xuy5VcYY80qK8i8B9W2viwDvAaGAASJs28YA44GBwEashNAO+NFBx6VUjpaUbNh35srt1j8Rx2KIirnJMM/vaed9lI8CR/JsWE1CygfSoEIgRfL7uDpklREiVt+EcY1hyd/gue/dYhIje64QwoBIY8xhABGZA3QB9tyjfC+sJADQFlhljLlo23YV0E5E1gIFjTEbbMunA13RhKByqSuxCWw9ful265+tx2O4Hm8N/FYiwJfQ4EBefjCZJzctJqn6k4zo/jcXR6yyLLACtH4ffnwDtn8L9Z5xdUTpsichBAEnUryPAh5Kq6CIVAAqAr/cZ9sg209UGsuVcnvGGE5cvHn7wW/EsRj2n72KMeAhUL1UQZ5sUPZ265+ygfkQY2BqR2vAtA7/cvUhKEdpOAB2fw8/jYDKrSCgpKsjui9HP1TuCcw3xiQ5qkIRGQQMAihf3j3uw6m8JTnZsPXEJWvYB1vzz/PXrLb/BXy9qF++MO1rl7YN/FYo7QlhtkyD439C56+gQAknH4HKNh4e1oio45vC8tegx0xXR3Rf9iSEk0DKSVvL2palpSeQstH0SaB5qm3X2paXtadOY8wEYAJY8yHYEa9STmOM4ZV521i07RQA5Yv406xqsdtNP6uVDMAzvYe/V8/AynchuBnUf84JUSunKlYVWoywHjDv/gFqdXV1RPdkT0LYDFQVkYpYJ+2ewF03w0SkOhAIrE+xeAXwTxEJtL1/DBhhjLkoIldEpBHWQ+U+wJjMH4ZSrjF2TSSLtp1iSIvK9G0STImATLT9//FNSIy1HkK6wYNHlQmNX7KSwfLXoeIjVq/mHCjdHhPGmERgKNbJfS8wzxizW0RGikjnFEV7AnNSNh21PUz+B1ZS2QyMvPWAGRgMfANEAofQB8rKzfy06wyfrDzAE/WDeP2xBzKXDPb/CHt+gEffcLterSoDPL2gy1dwM8Z6npBD6RSaSmXC3tNXeGr8n1QtGcDcQY3w885Ep7C4qzD2IWueg0G/gpc2Lc31fvkQfvs3PPOdNUKqk9g7hab79KlWKoc4fy2OAdPCKejnzcTeIZlLBgC/jIIrp6DTF5oM8opHXofi1WHpyxB7xdXR3EUTglIZEJ+YzIszIzh/LY4JfUIoUTCT4wVFhcPG/1rNEsuFOTZIlXN5+UKXsdYXgdXvpV/eyTQhKGUnYwzv/LCLzUdj+KTbg9QtWzhzFSUlWFNiBpSGVu86NkiV85UNhUaDrZnwjvzu6mjuoAlBKTtN+eMoc8NP8FLLKnR6sEzmK/rzSzi3Gzp+Yk2/qPKelm9DYDAsfgnib7g6mts0IShlh98ORDNq2R7a1irJK62rZb6iC4dg7b+gRieo3tFxASr34uNvdViLOQJrPnR1NLdpQlAqHYeirzFk9haqlQzg0+71Mj/KqDHWw0QvX2j/sWODVO6n4iMQ0g82jIOoCFdHA2hCUOq+Lt9IYOC0cHw8Pfimbyj5fbMw2su22XDkN2vAs4KlHRWicmdtRkKBUtZkOonxro5GE4JS95KYlMzQb7dwIuYGX/cOoWygf+YruxYNK/8O5RpBSH/HBancm18h6PQ5nNsDv//H1dFoQlDqXj5cvpffD55nVNfaNAzO4lADK96CuGtWnwM3mlJROUG1tlCnO/z+CZzd7dJQ9DdTqTTM3XycKX8c5fmmFenRMIuj7Eauhp3zoNmrUKK6YwJUuUu70eBX2Jp2MynRZWFoQlAqlU1HLvL2D7t4pFpx3uqQxRN4/HVY+goUrQrNXnNMgCr3yV8UOnwMp7bChrEuC0MTglIpnLh4g7/OjKBcoD9jetXHyzOLfyJr/gmXjtuGp/B1TJAqd6r1BFR/3PqdOR/pkhA0IShlcz0ukYHTw0lMSuabvqEUypfGRDYZcWqb1aSwQV8IbuqYIFXuJQId/2N9cVj8EiQnOz0ETQhKYc169srcbRw4e5WvnmlApeIFslZhUiIsGQb5i1tNC5WyR0ApaPtPa/a88ElO370mBKWAT1cdYOWes7zzeE0eqVY86xVuHA+nt0P7f0G+TI55pPKmes9CpRbWDGuXjjt115oQVJ63ePspvloTSc+G5ejXJDjrFcYcte4DV2sPNXPudIkqhxKxnjkZA0tetv51ErsSgoi0E5H9IhIpIsPvUaa7iOwRkd0iMtu2rIWIbEvxEysiXW3rporIkRTr6jnusJSyz46oS7zx3XbCgoswskttJKtTWBoDS18F8bAGr9MpMVVmBFaA1u/BoZ9h+7dO2226/fBFxBMYC7QBooDNIrLYGLMnRZmqwAigqTEmRkRKABhj1gD1bGWKYE2XuTJF9W8YY+Y76mCUyoizV2IZOD2cYgV8Gf9cA3y8HHDBvHO+9Ufc7l9QqGzW61N5V8OBsOt7a8rNyq0goGS279Kev4AwINIYc9gYEw/MAbqkKjMQGGuMiQEwxpxLo56ngR+NMTlnrFeVZ8UmJDFoRgRXYxP5pm8oRQs4oEnojYvw03AICoGwgVmvT+VtHh7WPMwJN2H5687ZpR1lgoATKd5H2ZalVA2oJiJ/iMgGEWmXRj09gdTXPh+KyA4R+UxEtJG2cgpjDMMX7GD7iUt81qMeNUo7aE6ClW9D7CXo9CV4ZHJaTaVSKlYVmg+HvYutWfaymaMeKnsBVYHmQC9goojcblohIqWBOsCKFNuMAKoDDYEiwJtpVSwig0QkXETCo6OjHRSuysvG/3qIH7ad4o22D9C2VinHVHr4V9g2C5q8BKVqO6ZOpQCaDIM+i6yZ1rKZPQnhJFAuxfuytmUpRQGLjTEJxpgjwAGsBHFLd2ChMSbh1gJjzGljiQOmYN2auosxZoIxJtQYE1q8uAOaA6o8bfWes3y8Yj+dHyzD4OaVHVNpwk1rnoPAivBomt9rlMo8Ty+o1Nwpu7InIWwGqopIRRHxwbr1szhVmR+wrg4QkWJYt5AOp1jfi1S3i2xXDYjVrKMrsCsT8Stlt/1nrvK3OVupE1SIfz9dN+stisAaw37RULh42BrG2Dtf1utUykXSbWVkjEkUkaFYt3s8gcnGmN0iMhIIN8Ystq17TET2AElYrYcuAIhIMNYVxq+pqp4lIsUBAbYBf3XMISl1t4vX43lh2mby+3oxoXcoft4OuMcfexnm9oYjv0Krd532LU6p7CLGiZ0esio0NNSEh2f/gxWVu8QnJtN70ka2nrjEvL80pl45B/QcvhwFs7rB+QPQ+Suo1yvrdSqVTUQkwhiT7kOILMwHqFTOZ4zhvcW72XjkIl/0rOeYZHBml5UM4q7Cs99B5ZZZr1OpHEATgsrVpq8/xrebjjO4eWW61EvdWjoTDq2xbhP5BsDzP2mLIpWr6FhGKtf6I/I8I5fuoXWNkrz+2ANZr3DbtzDraShcDgas1mSgch29QlC50pHz1xk8awtVihfg85718PDIQosiY+C3T2DNKKj4CPSYaU2OrlQuowlB5TpXYhMYMG0zHgLf9A2lgG8Wfs2TEmHZK7BlOtTtYT1A9vJxXLBK5SCaEFSukpRseGn2Vo5duMHMAQ9Rroh/5iuLuwbf9YPIVdZ8yC3f0dFLVa6mCUHlKqN/3MuvB6L55xN1aFSpaOYrunoWZneDMzvh8c8htL/jglQqh9KEoHKN78JPMPH3I/RrEswzD5XPfEXRB2DWU3D9PPSaA9XaOi5IpXIwTQgqV4g4dpG/L9zFw1WK8XbHGpmv6Nif8G0v8PSGfssgqIHjglQqh9Nmp8rtnbx0k7/MiCAoMB9jn2mAl2cmf613L4TpXSF/MXhhlSYDlefoFYJyazfiExkwLZy4xGTm9AmlkL93xisxBtaPhZV/h3KNoNe34F/E8cEqlcNpQlBuKznZ8Nq87ew/c4XJ/RpSpUSBTFSSBCvego1fQ43O8OQEHbFU5VmaEJTb+uLng/y46wxvd6xB8wdKZLyChJuwYADsWwqNhsBjo6xpC5XKozQhKLe0bMdpvvj5IN1CyvLCwxUzXsH1C/BtD2tawnajodGLjg9SKTejCUG5nV0nL/Pad9sIqRDIqCdqZ3yim4uHYebTcOUkdJ8GNbtkT6BKuZm8kRD++NIaqrjZa+Dt5+poVBacuxrLwOnhFM3vy9fPheDrlcGJbqIiYHZ3MEnQZzGUfyh7AlXKDeWNG6YXIuG3f8PXD1vtzJVbik1I4i8zIrh0I4EJfUIoHuCbsQr2LYepHcEnv9WsVJOBUnewKyGISDsR2S8ikSIy/B5luovIHhHZLSKzUyxPEpFttp/FKZZXFJGNtjrn2uZrzh6dv4TnFkBSHExpD0tetqY/VG7DGMNbC3ey9fglPuvxILXKZHC00U0TYe6zUKK6NXR1sarZE6hSbizdhCAinsBYoD1QE+glIjVTlakKjACaGmNqAS+nWH3TGFPP9tM5xfJ/AZ8ZY6oAMcALWTuUdFRpDYM3QOOhsGUafBUGe5dk6y6V40z8/TDfbznJq22q0a52afs3TE6GVe/B8teh6mNW7+MCmWiRpFQeYM8VQhgQaYw5bIyJB+YAqZ/CDQTGGmNiAIwx5+5XoVhPAVsC822LpgFdMxJ4pvjkh7YfwoCfoUBxmPsczHkWrpzO9l2rzPtl31k++nEfHeuW5qWWVezfMDEOvh8If3wOoc9Dj1nW74BSKk32JIQg4ESK91G2ZSlVA6qJyB8iskFE2qVY5yci4bblt076RYFLxpjE+9SZfYIawMA10Pp9iFwNY8Ng8yTr26TKUQ6evcqwb7dRq0xBPnn6QftbFN28BDOfgl3zodV70PFT8MwbbSiUyixHPVT2AqoCzYFewEQRuTWbeQVjTCjwDPC5iFTOSMUiMsiWUMKjo6MdFC7W4GUPvwIv/gll6sGyV2FqB2ukS5UjxFyPZ8D0cPL5eDKxTyj5fOxsUXTpBExuC8c3wJMTodmrOo+BUnawJyGcBMqleF/WtiylKGCxMSbBGHMEOICVIDDGnLT9exhYC9QHLgCFRcTrPnVi226CMSbUGBNavHhxuw4qQ4pWtpofdhkL5/bC103h139DYrzj96XslpCUzOBZWzh9OZb/9g6hdCE7h5M4vQO+aQ1XTlkNCep2z95AlcpF7EkIm4GqtlZBPkBPYHGqMj9gXR0gIsWwbiEdFpFAEfFNsbwpsMcYY4A1wNO27fsCi7J4LJknAvWfg6GboUYnWPMh/PcROLHJZSHldSOX7GH94QuMfrIODcoH2rdR5M9WKzIPT3h+BVR6NHuDVCqXSTch2O7zDwVWAHuBecaY3SIyUkRutRpaAVwQkT1YJ/o3jDEXgBpAuIhsty0fbYzZY9vmTeBVEYnEeqYwyZEHlikFSsDTk+GZeVZHtkmPwfI3rNfKaWZsOMaMDcf4y6OVeLJBWfs22jrT6nAWGGw1Ky1ZM91NlFJ3EuvLunsIDQ014eHhztlZ3FX4ZRRs/C8ULAMd/wMPtHfOvvOwPw+dp/ekTTxarTgT+4Ti6ZHOvX9jYO1o+HU0VGoB3aeDX0HnBKuUmxCRCNuz3PvKGz2VM8M3ANr/y+rR6lcIvu1pTbh+9ayrI8u1jl24zuBZW6hULD9f9KyXfjJISoBFQ61k8OAz8Ox3mgyUygJNCOkp1xAG/Qot3oZ9y2BsQ9gyw/pmqhzmamwCA6ZZV3/f9A0lwC+diW7irsLsHrBtJjz6JnQdZ7UcU0plmiYEe3j5wKNvWE1US9aGxUNhWie4cMjVkeUKScmGv83ZxpHz1xn3bAMqFE2n89iV09bD48NrofMYaPGWNitVygE0IWREsarQdyk8/rnVvHF8E/j9U+vWhcq0f6/Yxy/7zvFe51o0qVzs/oXP7YVJbeDCYXhmLjTo45wglcoDNCFklIcHhPaHoZussXF+/gAmtICTEa6OzC19vyWK//56mN6NKtC7UYX7Fz7yO0xqC0nx0H85VG3jnCCVyiM0IWRWQCnoMcMaH+fGeasz1E9vQdw1V0fmNrYcj2H4gp00rlSUdzul00x053yY+aT1ub+wyupdrpRyKE0IWVXjcRiyEUL6w4axMK4xHFzt6qhyvNOXbzJoegSlC/sx7tkGeHve41fRGFj3OSx4Aco2hBdWQGA6VxJKqUzRhOAIfoXg8U+h/0/WjGyznoIFA+H6eVdHliPdjE9i4PRwYhOS+KZPKIH57zEVRnKSNWz16veg1hPw3PeQz85ey0qpDNOE4EgVGsNf11nNIHcvhK8awvY52kQ1BWMMr8/fzu5TVxjTqz5VSwakXTD+BsztDZu/gSYvwVOTdfpTpbKZJgRH8/K1mkH+9XcoWgUW/gVmPAExR10dWY4w5pdIlu04zYj21WlR/R4T1Vw/bzXr3b8c2n8Mj42yHuYrpbKV/pVllxI1rAHWOnwCUeEwthH88SUkJaa/bS71067TfLrqAE82CGJgs0ppF7pwyHpAf3YX9JgJDw1ybpBK5WGaELKThweEDbQeOldqDqvegW9awuntro7M6Xafuswrc7dTv3xh/vlEnbQnujmxyepjEHfF6u9R43HnB6pUHqYJwRkKBUGvb6HbVKuX7YQWsOpd6z55HhB9NY6B08Ip7O/Nf3uH4OedxkQ3e5dYt4l8C1rNSss1dH6gSuVxmhCcRcRqKTN0E9R/Fv74AsY3toZfyMXiEpP468wILt6IZ2KfUEoEpPFgeON/rQfIJWtbQ1cXzdCkekopB9GE4Gz5Aq3xd/ouBfGE6V3gh8Fw46KrI3M4YwxvL9xFxLEY/tOtHrWDCt1ZIDkZVvwdfvw/eKAD9F0C+dMZukIplW00IbhKxWbw4h/Q7DXYMddqorpzfq5qojpp3RG+i4jib62q0rFu6TtXJsTCgudh/VfQcKDV69vH3zWBKqUAOxOCiLQTkf0iEikiw+9RpruI7BGR3SIy27asnoisty3bISI9UpSfKiJHRGSb7SfvjUXgnQ9avWsNr124vNUbd3Z3a5J4N7dm/zn+uXwv7WuX4m+tqt658sZFqynu7oXQZiR0+Nia9lIp5VLpzpgmIp7AAaANEIU1x3KvFFNhIiJVgXlAS2NMjIiUMMacE5FqgDHGHBSRMkAEUMMYc0lEpgJLjTHz7Q3WqTOmOVtyknUv/ZdR1vtW70DYILc8UUaeu8YTY/+gXBF/5r/YGH8fr/+tjDkGs562+mV0HQ91nr5nPUopx3DkjGlhQKQx5rAxJh6YA3RJVWYgMNYYEwNgjDln+/eAMeag7fUp4BxQ3P7DyEM8PKHxYBiyASo0gZ+GW00wz+52dWQZculGPAOmbcbX24OJfUPvTAantlnHdO0s9F6oyUCpHMaehBAEpLyHEWVbllI1oJqI/CEiG0SkXepKRCQM8AFSzirzoe1W0mci4pvB2HOnwuWtqSCf/Mb6Nv3fR+Dnf1j33HO4xKRkhs7eyqlLsfy3dwhBhfP9b+XBVTClA3j6wPMrIfhh1wWqlEqTox4qewFVgeZAL2CiiBS+tVJESgMzgP7GmGTb4hFAdaAhUAR4M62KRWSQiISLSHh0dLSDws3hRKBuNxi6Gep0h98/ga+bwtF1ro7svkYt28u6yPN8+ERtQioU+d+KiGnWdJdFK1t9DEpUd12QSql7sichnATKpXhf1rYspShgsTEmwRhzBOuZQ1UAESkILAP+bozZcGsDY8xpY4kDpmDdmrqLMWaCMSbUGBNavHgeu9vkXwSeGG/dXklKgKkdYfEwuHnJ1ZHdZfbG40z98ygDm1WkW6jt18UY+OVDWDIMKrewJrUpWPr+FSmlXMaehLAZqCoiFUXEB+gJLE5V5gesqwNEpBjWLaTDtvILgempHx7brhoQawyDrsCuLBxH7la5JQzeAE2GwdYZMDYMdv+QY5qobjh8gXcX7aL5A8UZ3r6GtTAx3upf8du/oX5v6DUHfO8xsqlSKkdINyEYYxKBocAKYC8wzxizW0RGikhnW7EVwAUR2QOsAd4wxlwAugOPAP3SaF46S0R2AjuBYsAohx5ZbuPjD4/9AwaugQIl4bu+MOdZuHLKpWGduHiDF2dGUKGoP1/2qo+nh0DsFZjdDbbPhuZvWR3xPL1dGqdSKn3pNjvNSXJ1s9OMSEq0Zmdb8xF4eEGb9yHkeacPEX0tLpGnxv3JmSux/DCkKRWL5bcS1KxuEL0POn1pDdOhlHIpRzY7VTmNpxc0/RsMXg9lQ2DZazClPZzb57QQkpMNL8/ZRmT0NcY928BKBmd3W0NXxxyDZ+ZpMlDKzWhCcGdFKkLvH6Dr13B+P3z9MKwdDYlx2b7rT1buZ/Xes7z7eE2aVikGh3+Fye2sDnb9l0OVVtkeg1LKsTQhuDsRqNcLhmyGWl1h7UfwdTM4viH9bTNp0baTjFt7iGceKk+fxhVg+1yY+RQUDLJGKy1dN9v2rZTKPpoQcosCxeGpb+DZ+ZBwAya3haWvWg94HWjbiUu8MX8HD1UswgedaiLrPoWFg6B8I3j+JyhcLv1KlFI5kiaE3KZqG6uJaqPBEDEFxj4E+5Y5pOozl2MZND2ckgV9Gf/Mg3j/+Br8PBLqdIPnFkC+wulXopTKsTQh5Ea+BaDdR/DCamv+hTnPwLw+cPVMpquMTUhi0IxwrsclMqlXLYos6W8lnIdfgScmgJeOPKKUu9OEkJuVDYG//GoNsb3/J/gqzBpGIoNNjY0xvDF/BztPXmZ813JU+7EHHFwJHf8Drd93enNXpVT20L/k3M7T25qE58U/rYe9S4bB1MfhfKTdVYxbe4gl20/xYTNfHvmtF0Tvh56zoeGAbAxcKeVsmhDyimJVrCkqO4+BszthfBP47WNriIn7WLn7DB+v2M8r1S7Qa+cAiL8O/ZbCA+2dFLhSylk0IeQlItCgj9VE9YH21mQ8E5pDVNq9v/eevsLLc7fx1xK7GHbydSRfERiwCoJCnBu3UsopNCHkRQElofs06Pkt3Iyxehf/OBzirt0ucuFaHAOmhTPQ+yfevPIRUvpBa+jqIpVcGLhSKjtpQsjLqneAIRutZwEbv4ZxjeDASuITkxk8I5yBNybwStIUpMbj0Hcx5C/q6oiVUtnIK/0iylmMMcQnJROfaPtJ8TrO9nP38qQ7lsUl3vlvfFLSXfXdUVdiMvFJnajmW5nXr3xFpdndWJXchL4mkQ6em+ChF6Hth245t7NSKmPydEIwxpCYbNI4YSalOmHeeWKOT0wmLuWJO40T860ycQm3lqV94o5PVZ+j+Hh64ONl+/H0wNfb465lAX5e+Hp54uvlgY9XY76RhrS6OJu2Z6fjZRKg7T+h8RCHxaSUytnyREJ4a+FO1h+6kOJEnER8kvXaUaN/e3nIHSdbHy8P24nW03rt6YG/jxeFby9PdYL28vjfyTmdk7lVxvPuk/ytbT098PCQTB5JKFwYAtejreEolFJ5Rp5ICEGF81E7qFCqE7XtZOuVxgk15ck5zZO87bWn5+3Xnpk+AedARStbP0qpPCVPJIQhLaq4OgSllMrx7GplJCLtRGS/iESKyPB7lOkuIntEZLeIzE6xvK+IHLT99E2xPEREdtrq/NI2t7JSSikXSfcKQUQ8gbFAGyAK2Cwii40xe1KUqQqMAJoaY2JEpIRteRHgPSAUMECEbdsYYDwwENgILAfaAT868uCUUkrZz54rhDAg0hhz2BgTD8wBuqQqMxAYazvRY4w5Z1veFlhljLloW7cKaCcipYGCxpgNxprUeTrQ1QHHo5RSKpPsSQhBwIkU76Nsy1KqBlQTkT9EZIOItEtn2yDb6/vVqZRSyokc9VDZC6gKNAfKAr+JSB1HVCwig4BBAOXLl3dElUoppdJgzxXCSSDlvIhlbctSigIWG2MSjDFHgANYCeJe2560vb5fnQAYYyYYY0KNMaHFixe3I1yllFKZYU9C2AxUFZGKIuID9AQWpyrzA9bVASJSDOsW0mFgBfCYiASKSCDwGLDCGHMauCIijWyti/oAixxxQEoppTIn3VtGxphEERmKdXL3BCYbY3aLyEgg3BizmP+d+PcAScAbxpgLACLyD6ykAjDSGHPR9nowMBXIh9W6SFsYKaWUC4lx1NgNTiAi0cCxTG5eDDjvwHAcRePKGI0rYzSujMmtcVUwxqR7z92tEkJWiEi4MSbU1XGkpnFljMaVMRpXxuT1uHQ+BKWUUoAmBKWUUjZ5KSFMcHUA96BxZYzGlTEaV8bk6bjyzDMEpZRS95eXrhCUUkrdR65LCOkN1S0iviIy17Z+o4gE55C4+olItIhss/0McEJMk0XknIjsusd6sQ1NHikiO0SkQXbHZGdczUXkcorP6l0nxVVORNakGOb9b2mUcfpnZmdcTv/MRMRPRDaJyHZbXB+kUcbpf492xuX0v8cU+/YUka0isjSNddn7eRljcs0PVse5Q0AlwAfYDtRMVWYw8LXtdU9gbg6Jqx/wlZM/r0eABsCue6zvgNVhUIBGwMYcEldzYKkLfr9KAw1srwOwhmhJ/f/o9M/Mzric/pnZPoMCttfeWEPdN0pVxhV/j/bE5fS/xxT7fhWYndb/V3Z/XrntCsGeobq7ANNsr+cDrZwwOY89cTmdMeY34OJ9ijURcSIAAALDSURBVHQBphvLBqCwbehyV8flEsaY08aYLbbXV4G93D1K7/+3d/asVQRRGH5eNIUgKKigoJJe8QNBFDtbJY0pUvhZ2oiVoI3/QAQbi6QQFUHUIkrs4i9QBAstUlhYBQImiCJEX4uZSLh4vYO4M/F6Hriwyx6Yl5d79uye3Z2p7lmhrupkDz7l3ZH8631oWT0fC3U1QdJO4AQw2SekU7+GrSCUTNX9M8b2MrAIbFkDugBO5TbDI0m7fnG8NqW6W3A03/I/l7Sn9uD5Vv0g6epyNU09+40uaOBZbn+8BuZJa6P09atiPpbogjb5eBO4Anzvc7xTv4atIPzLPAVGbe8jLSR0Z0D8/8wr0qf4+4FbpMkVqyFpI/AYuGx7qebYv2OAriae2f5m+wBpRuPDkvbWGHcQBbqq56Okk8C87Zddj9WPYSsIJVN1/4yRtB7YBCy01mV7wfbXvDsJHOpYUwklflbH9tLKLb/tGWBEaZbdzpE0Qjrp3rf95BchTTwbpKulZ3nMj8AL0lK5q2mRjwN1NcrHY8CYpPektvJxSfd6Yjr1a9gKQslU3dPAubw9Dsw6P6FpqaunzzxG6gO3Zho4m9+cOQIsOk1d3hRJ21f6ppIOk/7HnZ9E8phTwFvbN/qEVfesRFcLzyRtk7Q5b28grcv+riesej6W6GqRj7av2t5pe5R0jpi1fbonrFO//taKaWsCl03VPQXclTRHenA5sUZ0XZI0BixnXee71iXpAentk62SPgDXSQ/YsH0bmCG9NTMHfAYudK2pUNc4cFHSMvAFmKhQ1CFdwZ0B3uT+M8A1YPcqbS08K9HVwrMdwB1J60gF6KHtZ63zsVBX9XzsR02/4kvlIAiCABi+llEQBEHwh0RBCIIgCIAoCEEQBEEmCkIQBEEAREEIgiAIMlEQgiAIAiAKQhAEQZCJghAEQRAA8APiMXji8trkdwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(history.history['accuracy'])\n",
    "plt.plot(history.history['val_accuracy'])\n",
    "plt.legend(['training', 'valivation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_3 (Embedding)      (None, 200, 32)           960000    \n",
      "_________________________________________________________________\n",
      "unified_gru (UnifiedGRU)     (None, 200, 32)           6336      \n",
      "_________________________________________________________________\n",
      "unified_gru_1 (UnifiedGRU)   (None, 1)                 105       \n",
      "=================================================================\n",
      "Total params: 966,441\n",
      "Trainable params: 966,441\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "def lstm_model():\n",
    "    model = keras.Sequential([\n",
    "        layers.Embedding(input_dim=30000, output_dim=32, input_length=maxlen),\n",
    "        layers.GRU(32, return_sequences=True),\n",
    "        layers.GRU(1, activation='sigmoid', return_sequences=False)\n",
    "    ])\n",
    "    model.compile(optimizer=keras.optimizers.Adam(),\n",
    "                 loss=keras.losses.BinaryCrossentropy(),\n",
    "                 metrics=['accuracy'])\n",
    "    return model\n",
    "model = lstm_model()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 22500 samples, validate on 2500 samples\n",
      "Epoch 1/5\n",
      "22500/22500 [==============================] - 94s 4ms/sample - loss: 0.6498 - accuracy: 0.5847 - val_loss: 0.4840 - val_accuracy: 0.7664\n",
      "Epoch 2/5\n",
      "22500/22500 [==============================] - 98s 4ms/sample - loss: 0.4172 - accuracy: 0.8255 - val_loss: 0.4279 - val_accuracy: 0.8300\n",
      "Epoch 3/5\n",
      "22500/22500 [==============================] - 101s 5ms/sample - loss: 0.3179 - accuracy: 0.8825 - val_loss: 0.3918 - val_accuracy: 0.8472\n",
      "Epoch 4/5\n",
      "22500/22500 [==============================] - 99s 4ms/sample - loss: 0.3306 - accuracy: 0.8870 - val_loss: 0.3959 - val_accuracy: 0.8468\n",
      "Epoch 5/5\n",
      "22500/22500 [==============================] - 96s 4ms/sample - loss: 0.2607 - accuracy: 0.9120 - val_loss: 0.3849 - val_accuracy: 0.8532\n",
      "CPU times: user 14min 25s, sys: 38.9 s, total: 15min 4s\n",
      "Wall time: 8min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "history = model.fit(x_train, y_train, batch_size=64, epochs=5,validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VOXZ//HPlT2QhZCwJoQEDDsIGpFFKC4gVQsuiGj1gadVaq2i1tpif60LbZ9aRVvXWlTU8qjIo9XSlpalgoCgJQjKIiEhBEjCEggkLNlz/f6YIQ4xmEmYZLbr/XrlNTNn7nPOlQPznTv3nDm3qCrGGGOCQ4i3CzDGGNN2LPSNMSaIWOgbY0wQsdA3xpggYqFvjDFBxELfGGOCiIW+McYEEQt9Y4wJIhb6xhgTRMK8XUBDSUlJmpaW5u0yjDHGr2zcuPGwqnZqqp3PhX5aWhpZWVneLsMYY/yKiOxxp50N7xhjTBCx0DfGmCBioW+MMUHE58b0G1NdXU1BQQEVFRXeLiUgREVFkZKSQnh4uLdLMca0Mb8I/YKCAmJjY0lLS0NEvF2OX1NVjhw5QkFBAenp6d4uxxjTxvxieKeiooLExEQLfA8QERITE+2vJmOClF+EPmCB70F2LI0JXn4xvGOMMYGqvKqWLw+UsbWwlLCQEG65OLVV92eh76Zjx47x1ltvcddddzVrvauuuoq33nqLDh06nLXNww8/zNixY7niiivOtUxjjA87VVXD9iJHwG8pdNzmFp+gts4xV/kFqR0s9H3FsWPHePHFF78W+jU1NYSFnf0wLlmypMltz5kz55zrM8b4lpOVNWwrKmNLYSnbCkvZUljKruITOPOdpJhIBifHceXALgxKjmdQcjzd4qNavS4LfTfNnj2bXbt2MXToUMLDw4mKiiIhIYEdO3awc+dOrr32Wvbt20dFRQX33nsvM2fOBL66rMSJEyf49re/zSWXXMK6detITk7mr3/9K9HR0cyYMYNrrrmGKVOmkJaWxvTp0/nb3/5GdXU1//d//0e/fv0oLi7mlltuoaioiJEjR7J8+XI2btxIUlKSl4+MMeZ4RTXb6nvwpWwtLCXv8EnUGfBd4iIZ1D2eqwZ3Y3ByPINT4ukS1/oB3xi/C/3H/raN7UVlHt3mgO5xPPKdgd/Y5vHHH2fr1q1s3ryZVatWcfXVV7N169b60x7nz59Px44dKS8v56KLLuKGG24gMTHxjG3k5OTw9ttv8/LLLzN16lTee+89br311q/tKykpic8++4wXX3yRuXPn8sorr/DYY49x2WWX8dBDD/Gvf/2LV1991XMHwBjjttLyarYVlZ4xRLP78Mn657vFRzEoOZ5J5yczOCWOQcnxdI71TsA3xq3QF5GJwDNAKPCKqj7e4PmewHygE1AC3KqqBc7npgO/cDb9taq+4aHavWr48OFnnOf+7LPP8v777wOwb98+cnJyvhb66enpDB06FIALL7yQ/Pz8Rrd9/fXX17f5y1/+AsDatWvrtz9x4kQSEhI8+vsYY77u2KkqthaWsbXoqx78niOn6p9P7hDNoOQ4brggmYHJ8QxOjicpJtKLFTetydAXkVDgBWA8UABsEJHFqrrdpdlc4M+q+oaIXAb8FrhNRDoCjwCZgAIbnesebWnBTfXI20r79u3r769atYoVK1awfv162rVrx7hx4xo9Dz4y8qv/DKGhoZSXlze67dPtQkNDqamp8XDlxpjGHD1ZxRbn8Mw2Z8jvK/nqNZqSEM3g5HimZvZgsHMMvmP7CC9W3DLu9PSHA7mqmgcgIguByYBr6A8Afuy8vxL4wHn/SmC5qpY4110OTATePvfS21ZsbCzHjx9v9LnS0lISEhJo164dO3bs4JNPPvH4/kePHs2iRYv42c9+xrJlyzh6tMXvm8YEvSMnKut77o7bMgqPfRXwPRPbMSSlA7cM7+kM+Dg6tPO/gG+MO6GfDOxzeVwAXNygzefA9TiGgK4DYkUk8SzrJjfcgYjMBGYCpKa27ulKLZWYmMjo0aMZNGgQ0dHRdOnSpf65iRMn8tJLL9G/f3/69u3LiBEjPL7/Rx55hJtvvpkFCxYwcuRIunbtSmxsrMf3Y0ygKT5eWR/up4N+f+lXf4mnJ7Xngp4JTB/Vk0Hd4xmYHE98dOBel0r09MfLZ2sgMgWYqKq3Ox/fBlysqne7tOkOPA+kA6uBG4BBwO1AlKr+2tnul0C5qs492/4yMzO14SQqX375Jf3792/+bxdAKisrCQ0NJSwsjPXr1/PDH/6QzZs3t3h7dkxNIDpYVnHGGTRbCks5WFYJgIgj4Ac7x94Hdo9nYHIccVGBEfAislFVM5tq505PvxDo4fI4xbmsnqoW4ejpIyIxwA2qekxECoFxDdZd5cY+TQN79+5l6tSp1NXVERERwcsvv+ztkozxGlXlQFkFWwpK2epyqmTxcUfAhwj07hTDqN5JDHKG/IDuccRE+t0Jix7nzhHYAGSISDqOsJ8G3OLaQESSgBJVrQMewnEmD8BS4H9E5PSpJhOcz5tmysjIYNOmTd4uw5g2p6oUlToD3uWD1sMnqgBHwGd0jmVMRlJ9L35A9zjaRVjAN6bJo6KqNSJyN44ADwXmq+o2EZkDZKnqYhy9+d+KiOIY3vmRc90SEfkVjjcOgDmnP9Q1xpiGVJWCo+VnjMFvKyqj5KQj4ENDhIzOMVzat3P9t1gHdIsjOiLUy5X7D7feClV1CbCkwbKHXe6/C7x7lnXn81XP3xhjAEfA7y05xdbCsvox+K1FpRw7VQ1AWIjQp0ss4/t3YVBKPIO6x9G/WxxR4Rbw58L+/jHGtLq6OmVPyakzrkOztbCUsgrH91DCQ4W+XWP59qCu9WPwfbvGEhlmAe9pFvrGGLfV1NZRUVNHRXWt86eR+zWO++XVtew5fJKtRaVsKyzjeKUj4CPCQujfNZZrzu9ePwbfp0ssEWF+M72HX7PQbyUxMTGcOHGCoqIiZs2axbvvNjr69Y02b95MUVERV111FQCLFy9m+/btzJ4929PlGj9VV6f1IXs6fMudAVzpGsBVtWe0q3Rp5whqR5vKmjMDvOG2qmu/+RTvhiLDQujfLY5rhyUzKNlxHZo+XWIJD7WA9xYL/VbWvXv3FgU+OEI/KyurPvQnTZrEpEmTPFme8TBVpbK+J1znDE2XIK2ppaJBAJ9uV+nSrn69mrOEtPO5qpq6FtUpAlFhoUSFhxAVHkp0eCiR4c7HYaEkxYQRHRFKVJjLcme70/cdz4U4l53+cd1eCB3bRRBmAe9TLPTdNHv2bHr06MGPfvQjAB599FHCwsJYuXIlR48epbq6ml//+tdMnjz5jPXy8/O55ppr2Lp1KyNGjODVV19l4EDH9YPGjRvH3Llzqaur495776WiooLo6Ghee+010tPTefjhhykvL2ft2rU89NBDlJeXk5WVxfPPP09+fj7f+973OHz4MJ06deK1114jNTWVGTNmEBcXR1ZWFgcOHOCJJ55gypQpbX68Ak3uoRMsWJ/PnpJTzh5w3RkhXVFT6+wptyyEwdErbhiskeGhRIWFkNA+gm4uId0wZKNOrxsRSmSDdg2DOioihIjQEJs2M0j5X+j/czYc2OLZbXYdDN9+/Bub3HTTTdx33331ob9o0SKWLl3KrFmziIuL4/Dhw4wYMYJJkyad9cV00003sWjRIh577DH279/P/v37yczMpKysjDVr1hAWFsaKFSv4+c9/znvvvcecOXPqQx7g9ddfr9/WPffcw/Tp05k+fTrz589n1qxZfPCB45JH+/fvZ+3atezYsYNJkyZZ6LeQqvLp7hJeXp3Hv3ccIjIshL5dY4kKDyU+Opyo2MjGQ/qM3m+Is0fdIKQbtIsIDSEkxELYtD7/C30vGTZsGIcOHaKoqIji4mISEhLo2rUr999/P6tXryYkJITCwkIOHjxI165dG93G1KlTmTBhAo899hiLFi2qD+PS0lKmT59OTk4OIkJ1dXWT9axfv77+ssu33XYbP/3pT+ufu/baawkJCWHAgAEcPHjQA799cKmpreOfWw/w8po8vigopWP7CO67IoPbRvQk0ccvm2tMU/wv9JvokbemG2+8kXfffZcDBw5w00038eabb1JcXMzGjRsJDw8nLS2t0Usqn5acnExiYiJffPEF77zzDi+99BIAv/zlL7n00kt5//33yc/PZ9y4cedUp+slnJu6tpL5yonKGhZt2Mera3dTeKyc9KT2/Oa6QdxwQYqdG24Chv+FvhfddNNN3HHHHRw+fJiPPvqIRYsW0blzZ8LDw1m5ciV79uxxaxtPPPEEpaWlDBkyBHD09JOTHRcfdR3C+abLOY8aNYqFCxdy22238eabbzJmzJhz/wWD1MGyCl77OJ83P93D8Yoahqd15NFJA7m8X2cbcjEBxz5Wb4aBAwdy/PhxkpOT6datG9/97nfJyspi8ODB/PnPf6Zfv35NbmPKlCksXLiQqVOn1i/76U9/ykMPPcSwYcPOmDTl0ksvZfv27QwdOpR33nnnjO0899xzvPbaawwZMoQFCxbwzDPPeO4XDRI7DpTxwKLPueR3HzJv9S7GZnTi/btGsejOkYwf0MUC3wSkJi+t3Nbs0sptI1iPqaryce4R5q3JY/XOYqLDQ7npoh58b3Q6qYntvF2eMS3myUsrG+P3qmrq+PsXRcxbnceOA8fpFBvJg1f25bsXpwbMjEjGuMNC3wS0sopq3v50L699nM+BsgoyOsfwxJQhTB7a3a7rYoKS34S+qtqXSTzE14b0WkPhsXJeW7ubhRv2caKyhlG9E/ntDYMZ16eT/T8yQc0vQj8qKoojR46QmJhoL9hzpKocOXKEqKgob5fSKrYWljJvdR7/2LIfgO8M6cbtY3oxKDney5UZ4xv8IvRTUlIoKCiguLjY26UEhKioKFJSUrxdhsfU1Skf7Sxm3uo81ucdISYyjO+NTmPG6HSSO0R7uzxjfIpfhH54eDjp6eneLsP4mMqaWv66qYiX1+SRc+gEXeOi+PlV/Zg2PDVgJrs2xtP8IvSNcXX0ZBVvfrqH19ft4fCJSvp3i+MPNw3l6iHd7JK9xjTBQt/4jb1HTvHq2jwWZRVQXl3Lt/p0YubYXozqbZ/1GOMuC33j8zbtPcrLa/L419YDhIYIk4cmc/uYdPp1jfN2acb4HQt945Nq65QVXx7k5dV5ZO05SlxUGD/4Vm9mjEqjS1xgnnlkTFtwK/RFZCLwDBAKvKKqjzd4PhV4A+jgbDNbVZeISBrwJZDtbPqJqt7pmdJNIKqoruXdjQW8unY3uw+fJCUhmke+M4CpmT1oH2l9FBPgamsgtHX/nze5dREJBV4AxgMFwAYRWayq212a/QJYpKp/FJEBwBIgzfncLlUd6tmyTaA5fKKSBev3sOCTPZScrOL8lHiev2UYEwd2ten2jO+qq4XK41B1AipPOG+b+/gEVB133CZfAN9f1qolu/OWMhzIVdU8ABFZCEwGXENfgdMDrPFAkSeLNIFrV/EJXlmzm798VkBlTR1X9O/MHWN6MTy9o304azxPFapOfj1sW/q4pty9/UoIRMRCZAxExHx1G9P5zMcdW//UdHdCPxnY5/K4ALi4QZtHgWUicg/QHrjC5bl0EdkElAG/UNU1LS/XBAJVZUP+UeatzmPFlweJCAvhhgtS+P4l6ZzXOcbb5Rlfogo1lS3sQTfWoz6Bo4/qhvD2DUI6FuKSzwzpyFj3HodHO2aj9wGeGjy6GXhdVZ8SkZHAAhEZBOwHUlX1iIhcCHwgIgNVtcx1ZRGZCcwESE1N9VBJxtfU1NaxdNtB5q3J4/N9x0hoF86syzP4r5E9SfLlaQjLj8HR3XA0H6pOOZaJANLgljPvf21ZY+1cbhvb7hnLaGIbLd0XZ99/i/fl3FZNZQt71C6hXffVHBPfKDTy62HbLhE69PwqtBuG+NkeR7SHkMC8IJ87oV8I9HB5nOJc5ur7wEQAVV0vIlFAkqoeAiqdyzeKyC6gD3DGBfNVdR4wDxzX02/B72F82MnKGhZl7WP+x7vZV1JOWmI7fnXtIKZckEJ0hA+8sOrq4Ph+R7CX7D7z9mg+lB/1doWBSUIbD9+YLu73oF0fh9q3sN3hTuhvADJEJB1H2E8DbmnQZi9wOfC6iPQHooBiEekElKhqrYj0AjKAPI9Vb3zaobIKXl+Xz/9+soeyihoyeybw/64awPgBXQht61mpairh2N6vh3rJbji2B2pc5jaWUOjQAxLSYeB1jtuO6Y7byFhAHcMO9bdOZyxT95fVb8N1GY0sa6xdc/fVnO26u6+z/P6qEBZx9h51WJTPDHkEkyZDX1VrRORuYCmO0zHnq+o2EZkDZKnqYuAB4GURuR/H/4oZqqoiMhaYIyLVQB1wp6qWtNpvY3zCzoPHeXl1Hh9sLqSmTpk4sCu3j+nFhT0TWnfHp4dhXHvpJc7b0gLOGMsNb+8I8qQM6DPhzGCPT7FeowlYfjFdovF9qsq6XUd4eU0eq7KLiQoPYWqmYxrCtKT2ntlJXR2cONB4b/3o7q8Pw7TvdGaYu96272S9TBNQbLpE0yaqa+v4xxf7mbc6j+37y0iKieSB8X24dURPEtq3YBrCmko4tu/s4+tnG4YZcG2DcE9zDsUYY1xZ6JsWOV5RzcL/OD6c3V9aQe9O7fndDYOZPDSZqPAmPpytKD1Lbz2/kWGYdo4gTzwPzrvizGCP72HDMMY0k4W+aZaiY+W8vi6ftz/dy/HKGkb06shvrhvEuD6dCTn94awqHD/QeG+9ZDeUN/hYp12SI8RTR359KCamsw3DGONBFvrGLVsLS3llTR5//2I/CnxnUBJ3nh9Gv4gjUPJXWJZ/Zo/d9ZuKEuLolXdMhwGTbRjGGC+y0DdnpRWlZG36jHUbsig/mMvosGLuTTxGDw4SllMEO+u+ahzezhHgHXvBeZc77zvDvUOqDcMY4yMs9INZ/TBM/hlDMHUleVQV5xFVdZSLgIsAwqGuXRIh8emQMPLrZ8PEdLFhGGP8gIV+MKo6BVnzYd2zcOJg/WKVEI5HdmVHZSK51cM4FZPKwIHnkznsQsKT0gmJsklLjPF3FvrBpLocNr4Oa3/vCPv0b8HYBykO68bbuWG8urWG0mPCmIwk7hjTizEZSXalS2MCjIV+MKiugM/egDVPO77clDYGbnydrWED+eNHu/jnlv2ESB2ThqZw+yW9GNDdevTGBCoL/UBWUwmf/dkR9seLoOdomPIqpF1CwdFTXP/UR0SGhnDH2F7MGJVGt/hob1dsjGllFvqBqKYKNi2ANU9BWaHj/PfrXoL0sfUftj777xxQ+Od9Y0hJaOflgo0xbcVCP5DUVMHmNx1hX7oPelwMk1+AXuPOOLNmV/EJ3t1YwIxR6Rb4xgQZC/1AUFsNn78Nq590XD445SL4zjPQ+7JGT6N8evlOosJDuevS3l4o1hjjTRb6/qy2Br5Y6Aj7o/nQ/QK4+mnHNWrOctbN1sJS/vHFfu657Dzfnq3KGNMqLPT9UW0NbFkEHz3h+FJVt6FwyyLImNDkF6SeWpZNfHQ4t4/p1UbFGmN8iYW+P6mrhS3vwke/g5Jd0HUITHsb+n7brW/DbsgvYWV2MT+b2I/4aLssgjHByELfH9TVwta/OML+SA50GQw3vQn9rnb70geqypP/yqZTbCTTR/Vs5YKNMb7KQt+X1dXB9vdh1e/gcDZ0HghTF0C/ayAkpFmbWp1zmP/klzBn8kDaRdg/uzHByl79vqiuDr78qyPsi7+ETv3hxteh/+Rmhz04e/lLd5CSEM20i1I9X68xxm9Y6PuSujrY8XdY9Tgc2gZJfWHKfBhwXYvC/rR/bT3A1sIy5t54PhFhLd+OMcb/Wej7AlXY8Q9H2B/cAokZcP0rMOh6CGli6sEm1NYpc5dlc17nGK4bluyhgo0x/sqtbp+ITBSRbBHJFZHZjTyfKiIrRWSTiHwhIle5PPeQc71sEbnSk8X7PVXYsQT+NBbe+S5Un4Lr5sGPPoUhN55z4AO8v6mQXcUneWB8H0JD7IqZxgS7Jnv6IhIKvACMBwqADSKyWFW3uzT7BbBIVf8oIgOAJUCa8/40YCDQHVghIn1UtdbTv4hfUYWcZbDqt1C0yTERybUvweAbIdRzf3xV1dTxhxU7GZQcx8RBXT22XWOM/3InYYYDuaqaByAiC4HJgGvoK3D6erzxQJHz/mRgoapWArtFJNe5vfUeqN3/qELuCkfYF26EDj0d18YZMs2jYX/aOxv2UnC0nF9fO8iui2+MAdwL/WRgn8vjAuDiBm0eBZaJyD1Ae+AKl3U/abBu8A0sq8KuDx1hX7AB4lNh0nNw/s2tNndseVUtz36Yy/C0jnyrT6dW2Ycxxv94qnt5M/C6qj4lIiOBBSIyyN2VRWQmMBMgNTWATilUhbxVjrDf9ynE94Br/gBDvwthEa266zfW51N8vJIXbrnAevnGmHruhH4h0MPlcYpzmavvAxMBVHW9iEQBSW6ui6rOA+YBZGZmqrvF+7Tdq2Hlb2HvOohLdlwIbdhtrR72AGUV1fxx1S7G9e3E8PSOrb4/Y4z/cCf0NwAZIpKOI7CnAbc0aLMXuBx4XUT6A1FAMbAYeEtEnsbxQW4G8B8P1e6b8tc6wn7PWojtBlfNhQv+C8La7oqWr6zOo7S8mp9M6Ntm+zTG+IcmQ19Va0TkbmApEArMV9VtIjIHyFLVxcADwMsicj+OD3VnqKoC20RkEY4PfWuAHwXsmTt71sHK/4H8NRDTFb79BFwwHcKj2rSMwycqeWXtbq4e3I1ByfFtum9jjO9za0xfVZfgOA3TddnDLve3A6PPsu5vgN+cQ42+be+nsOp/HGP37TvDxMfhwhkQ7p35Zv+4ahcV1bXcP76PV/ZvjPFt9o3cltq3wRH2uz6E9p1gwm8g83sQ4b3pB4uOlbPgkz3ccEEK53WO8VodxhjfZaHfXIUbHWP2ucuhXRKM/xVc9H2IaO/tynjuwxxUlXuvyPB2KcYYH2Wh766iTY6wz1kK0R3hikfhojsg0jd61LsPn2RRVgG3jehpk50bY87KQr8p+z93hP3Of0J0Alz+MAyfCZGx3q7sDL9fvpOI0BCb7NwY840s9M/mwBbHVS93/B2i4uHSX8DFP4CouKbXbWPbi8pY/HkRd43rTefYtj1byBjjXyz0Gzq4zfEN2i//BpHxMO7nMOJOR/D7qKeXZxMbFcYPxlov3xjzzSz0Tzv0paNnv/0DiIyDb/0MRtwF0R28Xdk32rjnKCu+PMSDV/Ylvp1Ndm6M+WYW+sXZjrDf9j5ExMDYB2Hkjxzj9z7u9DSISTERzBiV5u1yjDF+IHhD/3AOfPQ72PKu43TLMT+GkXdDO/+5Vs3HuUf4JK+ER74zgPaRwftPaYxxX/AlxZFdzrD/PwiLhtH3wqhZ0D7R25U1y+lefvf4KG65OICuTGqMaVXBE/olefDRk/DFQgiLcvTqR98L7ZO8XVmLLNt+kM8LSnnihiFEhp37tIrGmOAQ+KFfshtWz4XP34bQCMeHs6PvhZjO3q6sxWrrlKeWZdMrqT3XXxB8c9IYY1oucEP/6B5YMxc2vwUhYY5z7EffB7FdvF3ZOVv8eSE7D57g+VuGERbq1tz2xhgDBGLoH9vnCPtN/wsSChfdDpfcD7GBMTF4dW0dv1+ew4BucVw1qJu3yzHG+JnACf2TR2Dlr+GzBSACF/6344ycuO7ersyjFmXtY2/JKV6bcREhITYNojGmeQIn9EVg+18ds1SN+THEp3i7Io+rqK7l2X/nkNkzgXF9bbJzY0zzBU7ot+sI92316vXsW9uC9Xs4WFbJM9OG2WTnxpgWCaxPAQM48I9XVPPiqlzGZCQxopd/fafAGOM7Aiv0A9ira3dz9FQ1D15pk50bY1rOQt8PlJys4pU1u5k4sCtDUnz7AnDGGN9moe8HXvpoFyeranhggk12bow5N26FvohMFJFsEckVkdmNPP97Edns/NkpIsdcnqt1eW6xJ4sPBgdKK3hjXT7XDUsmo4tvzdZljPE/TZ69IyKhwAvAeKAA2CAii1V1++k2qnq/S/t7gGEumyhX1aGeKzm4PPdhDnWq3H+F9fKNMefOnZ7+cCBXVfNUtQpYCEz+hvY3A297orhgt+fISd7ZsI9pF6XSo2PgnplkjGk77oR+MrDP5XGBc9nXiEhPIB340GVxlIhkicgnInJtiysNQn9YkUNYqHDPZed5uxRjTIDw9JezpgHvqmqty7KeqlooIr2AD0Vki6rucl1JRGYCMwFSU+3a8ADZB47zweZCZo7tRec4m+zcGOMZ7vT0C4EeLo9TnMsaM40GQzuqWui8zQNWceZ4/+k281Q1U1UzO3WyywsAPLUsm5iIMO60yc6NMR7kTuhvADJEJF1EInAE+9fOwhGRfkACsN5lWYKIRDrvJwGjge0N1zVn2rzvGMu2H+SOsb1IaB/h7XKMMQGkyeEdVa0RkbuBpUAoMF9Vt4nIHCBLVU+/AUwDFqqquqzeH/iTiNTheIN53PWsH9O4uUuz6dg+gu9dku7tUowxAcatMX1VXQIsabDs4QaPH21kvXXA4HOoL+isyz3M2tzD/OLq/sTYZOfGGA+zb+T6EFXlyWXZdIuP4tYRPb1djjEmAFno+5B/f3mITXuPMevyDKLCbbJzY4znWej7iLo6Ze6ybNIS2zHlwsCbAMYY4xss9H3E374oYseB49w/vg/hNtm5MaaVWLr4AMdk5zvp1zWW7wwJrDl9jTG+xULfB7y3sYD8I6f4yYS+Ntm5MaZVWeh7WUV1Lc/8O4dhqR24vH9nb5djjAlwFvpe9uane9lfWsGDV/a1yc6NMa3OQt+LTlTW8OLKXC45L4lRvZO8XY4xJghY6HvRa2t3c+RkFT+xyc6NMW3EQt9Ljp2qYt7qPMYP6MLQHjbZuTGmbVjoe8lLH+VxwiY7N8a0MQt9LzhUVsHr63Yz+fzu9Osa5+1yjDFBxELfC55fmUtNrXKfTXZujGljFvptbF/JKd7+z16mXtSDtKT23i7HGBNkLPTb2B9W5CAizLosw9ulGGOCkIV+G8o5eJz3NxUwfWRPusbbZOfGmLa6SbH2AAAO0klEQVRnod+Gnl6+k+jwUH447jxvl2KMCVIW+m3ki4Jj/HPrAW4f04uONtm5McZLLPTbyNxlO+nQLpzbx9hk58YY77HQbwOf5B1h9c5i7hrXm9iocG+XY4wJYhb6rUxVmbs0my5xkfzXyDRvl2OMCXJuhb6ITBSRbBHJFZHZjTz/exHZ7PzZKSLHXJ6bLiI5zp/pnizeH6zKLiZrz1HuucwmOzfGeF9YUw1EJBR4ARgPFAAbRGSxqm4/3UZV73dpfw8wzHm/I/AIkAkosNG57lGP/hY+qq5OeXJpNqkd2zE1s4e3yzHGGLd6+sOBXFXNU9UqYCEw+Rva3wy87bx/JbBcVUucQb8cmHguBfuTJVv3s31/GfePzyAizEbSjDHe504SJQP7XB4XOJd9jYj0BNKBD5uzrojMFJEsEckqLi52p26fV1Nbx9PLd9KnSwyTzm/0cBljTJvzdPdzGvCuqtY2ZyVVnaeqmaqa2alTJw+X5B1/2VRIXvFJHpjQl1Cb7NwY4yPcCf1CwHVAOsW5rDHT+Gpop7nrBozKmlqeWZHD+SnxTBjQxdvlGGNMPXdCfwOQISLpIhKBI9gXN2wkIv2ABGC9y+KlwAQRSRCRBGCCc1lAe/vTvRQeK+fBK/vZZOfGGJ/S5Nk7qlojInfjCOtQYL6qbhOROUCWqp5+A5gGLFRVdVm3RER+heONA2COqpZ49lfwLaeqanh+ZS4jeyUy+rxEb5djjDFnaDL0AVR1CbCkwbKHGzx+9Czrzgfmt7A+v/Pax/kcPlHFn27ra718Y4zPsfMIPaj0VDV/+mgXV/TvzIU9E7xdjjHGfI2FvgfNW7OLsooaHpjQ19ulGGNMoyz0PaT4eCXz1+Yz6fzu9O9mk50bY3yThb6HvLAyl6raOu4fb5OdG2N8l4W+BxQcPcVbn+7lxgtTSLfJzo0xPsxC3wOe/XcOALMut8nOjTG+zUL/HO0qPsG7Gwu4dURPuneI9nY5xhjzjSz0z9HTy3cSFR7KXZf29nYpxhjTJAv9c7C1sJR/fLGf71+STlJMpLfLMcaYJlnon4OnlmUTHx3O7WN6ebsUY4xxi4V+C23IL2FldjF3fqs38dE22bkxxj9Y6LeAqvLkv7LpFBvJ9FE9vV2OMca4zUK/BVbnHOY/+SXcc9l5tItw65p1xhjjEyz0m0lVmbs0m5SEaKZdlOrtcowxplks9Jtp6bYDbCks5b4r+thk58YYv2Op1Qy1dcrcZTs5r3MM1w2zyc6NMf7HQr8ZPthUSO6hEzwwvo9Ndm6M8UsW+m6qqqnj9yt2Mjg5nomDunq7HGOMaRELfTe9s2EvBUfL+cmVNg2iMcZ/Wei7obyqlmc/zGV4ekfGZiR5uxxjjGkxC303vLE+n+LjlTxovXxjjJ9zK/RFZKKIZItIrojMPkubqSKyXUS2ichbLstrRWSz82expwpvK2UV1fxx1S4u7duJi9I6erscY4w5J01+nVREQoEXgPFAAbBBRBar6naXNhnAQ8BoVT0qIp1dNlGuqkM9XHebeWV1HqXl1TbZuTEmILjT0x8O5KpqnqpWAQuByQ3a3AG8oKpHAVT1kGfL9I7DJyp5Ze1urh7SjUHJ8d4uxxhjzpk7oZ8M7HN5XOBc5qoP0EdEPhaRT0RkostzUSKS5Vx+bWM7EJGZzjZZxcXFzfoFWtMfV+2iorqWH9tk58aYAOGpq4WFARnAOCAFWC0ig1X1GNBTVQtFpBfwoYhsUdVdriur6jxgHkBmZqZ6qKZzUnSsnAWf7GHKhSn07hTj7XKMMcYj3OnpFwI9XB6nOJe5KgAWq2q1qu4GduJ4E0BVC523ecAqYNg51twmnvswB9QmOzfGBBZ3Qn8DkCEi6SISAUwDGp6F8wGOXj4ikoRjuCdPRBJEJNJl+WhgOz5u9+GTLMoq4JaLU0lJaOftcowxxmOaHN5R1RoRuRtYCoQC81V1m4jMAbJUdbHzuQkish2oBR5U1SMiMgr4k4jU4XiDedz1rB9f9fvlO4kIDbHJzo0xAcetMX1VXQIsabDsYZf7CvzY+ePaZh0w+NzLbDvbi8pY/HkRd43rTefYKG+XY4wxHmXfyG3g6eXZxEaF8YOx1ss3xgQeC30XG/ccZcWXhxyTnbezyc6NMYHHQt9JVXly6Q6SYiKYMSrN2+UYY0yrsNB3+jj3CJ/klfCjS8+jfaRNdm6MCUwW+nzVy+8eH8UtF9tk58aYwGWhDyzffpDPCxyTnUeGhXq7HGOMaTVBH/q1dcpTy3bSK6k9119gk50bYwJb0If+3z4vIvvgcX48oQ9hoUF/OIwxAS6oU666to6nl+9kQLc4rhrUzdvlGGNMqwvq0F+UtY+9Jad48Mq+hITYNIjGmMAXtKFfUV3Ls//OIbNnAuP6dvJ2OcYY0yaCNvQXrN/DwTKb7NwYE1yCMvSPV1Tz4qpcxvbpxMW9Er1djjHGtJmgDP1X1+7m6KlqHrTJzo0xQSboQr/kZBWvrNnNtwd1ZXCKTXZujAkuQRf6L320i1NVNTbZuTEmKAVV6B8oreCNdflcNyyFjC6x3i7HGGPaXFCF/nMf5lCnyn1X2GTnxpjgFDShv+fISd7ZsI+bh6fSo6NNdm6MCU5BE/p/WJFDWKhw96XnebsUY4zxmqAI/ewDx/lgcyEzRqXTOc4mOzfGBC+3Ql9EJopItojkisjss7SZKiLbRWSbiLzlsny6iOQ4f6Z7qvDmeGpZNjERYdz5rV7e2L0xxviMJucFFJFQ4AVgPFAAbBCRxaq63aVNBvAQMFpVj4pIZ+fyjsAjQCagwEbnukc9/6s0bvO+YyzbfpAfj+9Dh3YRbbVbY4zxSe709IcDuaqap6pVwEJgcoM2dwAvnA5zVT3kXH4lsFxVS5zPLQcmeqZ098xdmk3H9hF875L0ttytMcb4JHdCPxnY5/K4wLnMVR+gj4h8LCKfiMjEZqzbatblHmZt7mHuGtebGJvs3Bhjmh7eacZ2MoBxQAqwWkQGu7uyiMwEZgKkpnpmYnJV5cll2XSLj+LWET09sk1jjPF37vT0C4EeLo9TnMtcFQCLVbVaVXcDO3G8CbizLqo6T1UzVTWzUyfPXNv+wx2H2LT3GLMuzyAq3CY7N8YYcC/0NwAZIpIuIhHANGBxgzYf4OjlIyJJOIZ78oClwAQRSRCRBGCCc1mrqqtTnlyaTVpiO6ZcmNLauzPGGL/R5PCOqtaIyN04wjoUmK+q20RkDpClqov5Kty3A7XAg6p6BEBEfoXjjQNgjqqWtMYv4urvW/az48Bxnpk2lHCb7NwYY+qJqnq7hjNkZmZqVlZWi9evrq1j/NMfERUeypJZY2zuW2NMUBCRjaqa2VS7gOsGv7exgPwjp/jJBJvs3BhjGgqo0K+oruWZf+cwLLUDl/fv7O1yjDHG5wRU6L/56V72l1bYZOfGGHMWARP6JypreHFlLpecl8So3kneLscYY3xSwHxN9VRlDReldeTOcb29XYoxxvisgAn9znFRvHTbhd4uwxhjfFrADO8YY4xpmoW+McYEEQt9Y4wJIhb6xhgTRCz0jTEmiFjoG2NMELHQN8aYIGKhb4wxQcTnLq0sIsXAnnPYRBJw2EPleJLV1TxWV/NYXc0TiHX1VNUmpx70udA/VyKS5c41pdua1dU8VlfzWF3NE8x12fCOMcYEEQt9Y4wJIoEY+vO8XcBZWF3NY3U1j9XVPEFbV8CN6RtjjDm7QOzpG2OMOQu/DH0RmSgi2SKSKyKzG3k+UkTecT7/qYik+UhdM0SkWEQ2O39ub6O65ovIIRHZepbnRUSeddb9hYhc4CN1jRORUpfj9XAb1dVDRFaKyHYR2SYi9zbSps2PmZt1tfkxE5EoEfmPiHzurOuxRtq0+WvSzbq88pp07jtURDaJyN8bea71jpeq+tUPEArsAnoBEcDnwIAGbe4CXnLenwa84yN1zQCe98IxGwtcAGw9y/NXAf8EBBgBfOojdY0D/u6F49UNuMB5PxbY2ci/ZZsfMzfravNj5jwGMc774cCnwIgGbbzxmnSnLq+8Jp37/jHwVmP/Xq15vPyxpz8cyFXVPFWtAhYCkxu0mQy84bz/LnC5tP5M6e7U5RWquhoo+YYmk4E/q8MnQAcR6eYDdXmFqu5X1c+c948DXwLJDZq1+TFzs6425zwGJ5wPw50/DT8sbPPXpJt1eYWIpABXA6+cpUmrHS9/DP1kYJ/L4wK+/h+/vo2q1gClQKIP1AVwg3M44F0R6dHKNbnL3dq9YaTzz/N/isjAtt6588/qYTh6ia68esy+oS7wwjFzDlVsBg4By1X1rMerDV+T7tQF3nlN/gH4KVB3ludb7Xj5Y+j7s78Baao6BFjOV+/kpnGf4fhq+fnAc8AHbblzEYkB3gPuU9Wyttz3N2miLq8cM1WtVdWhQAowXEQGtcV+m+JGXW3+mhSRa4BDqrqxtffVGH8M/ULA9d04xbms0TYiEgbEA0e8XZeqHlHVSufDVwBfmcndnWPa5lS17PSf56q6BAgXkaS22LeIhOMI1jdV9S+NNPHKMWuqLm8eM+c+jwErgYkNnvLGa7LJurz0mhwNTBKRfBzDwJeJyP82aNNqx8sfQ38DkCEi6SISgeNDjsUN2iwGpjvvTwE+VOcnIt6sq8GY7yQcY7K+YDHwX84zUkYApaq639tFiUjX0+OYIjIcx//XVg8K5z5fBb5U1afP0qzNj5k7dXnjmIlIJxHp4LwfDYwHdjRo1uavSXfq8sZrUlUfUtUUVU3DkRMfquqtDZq12vEK88RG2pKq1ojI3cBSHGfMzFfVbSIyB8hS1cU4XhgLRCQXxweF03ykrlkiMgmocdY1o7XrAhCRt3Gc1ZEkIgXAIzg+1EJVXwKW4DgbJRc4Bfy3j9Q1BfihiNQA5cC0NnjzBkdP7DZgi3M8GODnQKpLbd44Zu7U5Y1j1g14Q0RCcbzJLFLVv3v7NelmXV55TTamrY6XfSPXGGOCiD8O7xhjjGkhC31jjAkiFvrGGBNELPSNMSaIWOgbY0wQsdA3xpggYqFvjDFBxELfGGOCyP8HZUXEnJmMEwkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history.history['accuracy'])\n",
    "plt.plot(history.history['val_accuracy'])\n",
    "plt.legend(['training', 'valivation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.CuDNN LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "module 'tensorflow.python.keras.api._v2.keras.layers' has no attribute 'CuDNNLSTM'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-95247c425d78>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      9\u001b[0m                  metrics=['accuracy'])\n\u001b[1;32m     10\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlstm_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msummary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-20-95247c425d78>\u001b[0m in \u001b[0;36mlstm_model\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m     model = keras.Sequential([\n\u001b[1;32m      3\u001b[0m         \u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEmbedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmaxlen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m         \u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCuDNNLSTM\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_sequences\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m         \u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCuDNNLSTM\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mactivation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'sigmoid'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_sequences\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     ])\n",
      "\u001b[0;31mAttributeError\u001b[0m: module 'tensorflow.python.keras.api._v2.keras.layers' has no attribute 'CuDNNLSTM'"
     ]
    }
   ],
   "source": [
    "def lstm_model():\n",
    "    model = keras.Sequential([\n",
    "        layers.Embedding(input_dim=30000, output_dim=32, input_length=maxlen),\n",
    "        layers.CuDNNLSTM(32, return_sequences=True),\n",
    "        layers.CuDNNLSTM(1, activation='sigmoid', return_sequences=False)\n",
    "    ])\n",
    "    model.compile(optimizer=keras.optimizers.Adam(),\n",
    "                 loss=keras.losses.BinaryCrossentropy(),\n",
    "                 metrics=['accuracy'])\n",
    "    return model\n",
    "model = lstm_model()\n",
    "model.summary()\n",
    "# tf2版没有"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "history = model.fit(x_train, y_train, batch_size=64, epochs=5,validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(history.history['accuracy'])\n",
    "plt.plot(history.history['val_accuracy'])\n",
    "plt.legend(['training', 'valivation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.CuDNN GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lstm_model():\n",
    "    model = keras.Sequential([\n",
    "        layers.Embedding(input_dim=30000, output_dim=32, input_length=maxlen),\n",
    "        layers.CuDNNGRU(32, return_sequences=True),\n",
    "        layers.CuDNNGRU(1, activation='sigmoid', return_sequences=False)\n",
    "    ])\n",
    "    model.compile(optimizer=keras.optimizers.Adam(),\n",
    "                 loss=keras.losses.BinaryCrossentropy(),\n",
    "                 metrics=['accuracy'])\n",
    "    return model\n",
    "model = lstm_model()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "history = model.fit(x_train, y_train, batch_size=64, epochs=5,validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(history.history['accuracy'])\n",
    "plt.plot(history.history['val_accuracy'])\n",
    "plt.legend(['training', 'valivation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
